# -*- coding: utf-8 -*-
import scrapy
from csdn.items import CsdnItem
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Rule
from scrapy_redis.spiders import RedisCrawlSpider
from datetime import datetime

class HomeSpider(RedisCrawlSpider):
    name = 'home'
    allowed_domains = ['blog.csdn.net']
    #start_urls = ['https://blog.csdn.net/']
    redis_key = 'csdn:start_url'
    link = LinkExtractor(allow=r'/article/details/\d+')
    rules = (
        # 规则解析器：可以将连接提取器提取到的所有连接表示的页面进行指定规则（回调函数）的解析
        Rule(link, callback='parse_item', follow=True),
    )

    def parse_item(self,response):
        item=CsdnItem()
        item['title']=response.xpath('//*[@id="articleContentId"]/text()').extract_first().strip()
        content=response.xpath('//*[@id="article_content"]|//article|//*[@id="article"]').get()
        if not content:
            item['content']=''
        else:
            item['content']=content.strip()
        item['href'] = response.url
        item['create_time'] = datetime.now()
        item['nickName'] = response.xpath('//a[@class="follow-nickName"]/text()|//div[@class="bar-content"]/a[1]/text()').extract_first()
        item['time'] = response.xpath('//div[@class="bar-content"]/span[@class="time"]/text()|//span[@class="time"]/text()').extract_first()
        tag=response.xpath('//a[@class="tag-link"]/text()').extract_first()
        item['tag'] =tag.strip() if tag else ''
        return item
